# -*- coding: utf-8 -*-
import scrapy
'''scrapy crawl myspider -a category=electronics'''

class ExampleComSpiderSpider(scrapy.Spider):
    name = 'example.com.spider'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com/']

    def __init__(self, category=None, *args, **kwargs):
        super(ExampleComSpiderSpider, self).__init__(*args, **kwargs)
        self.log(message='arg=%s' % category)
        self.category = category
        self.start_urls = ['http://www.example.com/categories/%s' % category]

    def start_requests(self):
        self.log(message='category_test=%s ' % self.category)

    def parse(self, response):
        """demo"""
        for demo in response.xpath("//h3").getall():
            yield {"title": demo}

        for href in response.xpath('//a/@href').getall():
            yield scrapy.Request(response.urljoin(href), self.parse)
